# -*- coding: utf-8 -*-
import urllib.request
from scrapy.selector import lxmlsel


# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://doc.scrapy.org/en/latest/topics/item-pipeline.html
class ScrapyloginPipeline(object):
    def process_item(self, item, spider):
        item = dict(item)
        projectsUrl = item["projectsUrl"]
        
        opener = urllib.request.build_opener()
        opener.addheaders=[
            ('User-Agent','Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36'),
            ('Accept-Language', 'zh-CN,zh;q=0.8')
        ]
        urllib.request.install_opener(opener)
        
        projectsPageContent = urllib.request.urlopen("https://gitee.com/"+projectsUrl)
        ppc = str(projectsPageContent.read(),encoding="utf-8")#ISO-8859-1
        
        selector = lxmlsel.Selector(text = ppc)
        title = selector.xpath("/html/head/title/text()").extract()
        print("=========================")
        print("=========================")
        print("title = ",title)
        
        #项目名称
        pjtName = selector.xpath("//span[@class='project-title']/a[@class='repository']/@title").extract()
        #项目URL
        pjtUrls = selector.xpath("//span[@class='project-title']/a[@class='repository']/@href").extract()
        #项目语言
        pjtLanguage = selector.xpath("//a[@id='project-language']/text()").extract()
        #项目描述
        pjtDescription = selector.xpath("//p[@class='description']/text()").extract()
        #关注相关数据
        pjtWatchNum = selector.xpath("//ul[@class='repo-stats float-right']/li[@class='watch']/a[2]/text()").extract()
        pjtStarNum = selector.xpath("//ul[@class='repo-stats float-right']/li[@class='star']/a[2]/text()").extract()
        pjtForkNum = selector.xpath("//ul[@class='repo-stats float-right']/li[@class='fork']/a[2]/text()").extract()
        
        if len(pjtName)>0:
            for i in range(0,len(pjtName)):
                name = pjtName[i]
                url = pjtUrls[i]
                language = pjtLanguage[i]
                description = pjtDescription[i].strip()
                watchNum = pjtWatchNum[i].strip()
                starNum = pjtStarNum[i].strip()
                forkNum = pjtForkNum[i].strip()
                print("\t*********项目："+str(i+1)+"**********")
                print("\tname = %s \r\n\turl = %s \r\n\tlanguage = %s \r\n\tdescription = %s \r\n\twatchNum = %s \r\n\tstarNum = %s \r\n\tforkNum = %s"
                      %(name,"https://gitee.com/"+url,language,description,watchNum,starNum,forkNum))
        else:
            print("该用户下没有新建项目")
        
        print("=========================")
        print("=========================")
        
        return item
